from datetime import datetime

import json
import scrapy

from books_rank_crawler.items import KongfzBookItem
from books_rank_crawler.settings import FEED_EXPORT_FIELDS_KONGFZ


class KongfzRankSpider(scrapy.Spider):
    name = "kongfz_rank"
    allowed_domains = ["kongfz.com"]
    custom_settings = {
        'FEED_EXPORT_FIELDS': FEED_EXPORT_FIELDS_KONGFZ
    }

    def start_requests(self):
        yield scrapy.Request(
            url="https://item.kongfz.com/api/pc/getRecentlyPublishedDetail?page=1&pageSize=100")  # 向url发送HTTP请求

    def parse(self, response, **kwargs):
        html = response.text  # 获取html
        book_item = KongfzBookItem()  # 构建BookItem对象
        if json.loads(html).get('result'):
            result = json.loads(html).get('result')  # 获取result
            data = result.get("data")  # 获取data数据
            rank = 1  # 初始化排名
            for item in data:
                book_item["rank"] = rank  # 排名
                book_item["author"] = item["authorArr"][0].get("name")  # 作者
                book_item["url"] = item["authorArr"][0].get("url")  # 作者url
                book_item["book_name"] = item["bookName"]  # 书名
                book_item["content_introduction"] = item["contentIntroduction"]  # 简介
                book_item["img_url"] = item["imgUrl"]  # 图片
                book_item["isbn"] = item["isbn"]  # isbn号
                book_item["new_min_price"] = item["newMinPrice"]  # 新书价格
                book_item["old_min_price"] = item["oldMinPrice"]  # 旧书价格
                book_item["press"] = item["press"]  # 出版社
                book_item["press_url"] = item["pressUrl"]  # 出版社url
                book_item["price"] = item["price"]  # 定价
                book_item["pub_date"] = item["pubDate"]  # 出版时间
                rank += 1
                book_item["crawl_time"] = datetime.now().strftime("%Y%m%d")  # 抓取时间
                book_item["crawl_url"] = response.url  # 抓取url
                yield book_item
